{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports\n",
    "\n",
    "import urllib.request, json \n",
    "\n",
    "from fuzzywuzzy import process\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Organize data\n",
    "\n",
    "places = sorted(list(set(['Alexandria', 'Amiternum', 'Andes', 'Antiochia', 'Aponus', 'Aquinum', 'Arles', 'Arpinum', 'Asisium ', 'Augusta Taurinorum ', 'Autun', 'Bilbilis', 'Bithynia ', 'Brigantium ', 'Calagurris', 'Carthage', 'Cisalpine Gaul', 'Comum', 'Corduba', 'Ebro Valley, Spain', 'Gadiz ', 'Hippo Regius ', 'Iliberri', 'Milan', 'Neapolis', 'North Africa', 'Parthenope/Neapolis', 'Patavium', 'Pydna', 'Roma', 'Rudiae', 'Sarsina', 'Stridon', 'Sulmo', 'Thagaste ', 'Ticinum', 'Tomis ', 'Velitrae ', 'Venusia', 'Verona', 'Verona'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get Pleiades locations from github\n",
    "\n",
    "name_index_url = \"https://raw.githubusercontent.com/ryanfb/pleiades-geojson/gh-pages/name_index.json\"\n",
    "\n",
    "with urllib.request.urlopen(name_index_url) as url:\n",
    "    pleiades_names = json.loads(url.read().decode())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make list of possible choices for fuzzy matching\n",
    "\n",
    "choices = [pleiades_name[0] for pleiades_name in pleiades_names]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make dictionary from pleiades_names for easier lookup\n",
    "\n",
    "pleiades_dict = dict(pleiades_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 40/40 [04:05<00:00,  6.86s/it]\n"
     ]
    }
   ],
   "source": [
    "# Fuzzy match and retrieve geojson from Pleiades\n",
    "\n",
    "# Separate this into two routines: 1. fuzzy matching, 2. retrieve geojson\n",
    "\n",
    "geojson_base_url = 'https://raw.githubusercontent.com/ryanfb/pleiades-geojson/gh-pages/geojson/'\n",
    "\n",
    "rows = []\n",
    "\n",
    "for place in tqdm(places):\n",
    "    top = process.extractOne(place, choices)\n",
    "    \n",
    "    # See development notes below; could handle better\n",
    "    if top[0]=='Alexandria':\n",
    "        top_id = '727070'\n",
    "    else:\n",
    "        top_id = pleiades_dict[top[0]]\n",
    "        \n",
    "    with urllib.request.urlopen(f'{geojson_base_url}{top_id}.geojson') as url:\n",
    "        geojson = json.loads(url.read().decode())\n",
    "        if geojson['features'][0]['geometry']:\n",
    "            lat, long = geojson['features'][0]['geometry']['coordinates']\n",
    "        else:\n",
    "            lat, long = None, None\n",
    "            \n",
    "    row = [place, top[0], top_id, lat, long]\n",
    "    rows.append(row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>place</th>\n",
       "      <th>match</th>\n",
       "      <th>pleiades_id</th>\n",
       "      <th>lat</th>\n",
       "      <th>long</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Alexandria</td>\n",
       "      <td>Alexandria</td>\n",
       "      <td>727070</td>\n",
       "      <td>29.909773</td>\n",
       "      <td>31.201435</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Amiternum</td>\n",
       "      <td>Amiternum</td>\n",
       "      <td>413013</td>\n",
       "      <td>13.306221</td>\n",
       "      <td>42.400467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Andes</td>\n",
       "      <td>Andes</td>\n",
       "      <td>383570</td>\n",
       "      <td>10.824999</td>\n",
       "      <td>45.125167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Antiochia</td>\n",
       "      <td>Antiochia</td>\n",
       "      <td>897709</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Aponus</td>\n",
       "      <td>Aponus</td>\n",
       "      <td>393376</td>\n",
       "      <td>11.783797</td>\n",
       "      <td>45.331858</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Aquinum</td>\n",
       "      <td>Aquinum</td>\n",
       "      <td>432695</td>\n",
       "      <td>13.705626</td>\n",
       "      <td>41.492208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Arles</td>\n",
       "      <td>Arles</td>\n",
       "      <td>148217</td>\n",
       "      <td>4.630799</td>\n",
       "      <td>43.677616</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Arpinum</td>\n",
       "      <td>Arpinum</td>\n",
       "      <td>432700</td>\n",
       "      <td>13.613727</td>\n",
       "      <td>41.648358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Asisium</td>\n",
       "      <td>Asisium</td>\n",
       "      <td>413037</td>\n",
       "      <td>12.618957</td>\n",
       "      <td>43.070619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Augusta Taurinorum</td>\n",
       "      <td>Augusta Taurinorum</td>\n",
       "      <td>383580</td>\n",
       "      <td>7.685998</td>\n",
       "      <td>45.071779</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Autun</td>\n",
       "      <td>Autun</td>\n",
       "      <td>177460</td>\n",
       "      <td>4.299177</td>\n",
       "      <td>46.947240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Bilbilis</td>\n",
       "      <td>Bilbilis</td>\n",
       "      <td>246247</td>\n",
       "      <td>-1.603219</td>\n",
       "      <td>41.382010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Bithynia</td>\n",
       "      <td>Bithynia</td>\n",
       "      <td>991389</td>\n",
       "      <td>32.500000</td>\n",
       "      <td>42.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Brigantium</td>\n",
       "      <td>Brigantium</td>\n",
       "      <td>236465</td>\n",
       "      <td>-8.395968</td>\n",
       "      <td>43.371485</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Calagurris</td>\n",
       "      <td>La(...)</td>\n",
       "      <td>711243</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Carthage</td>\n",
       "      <td>Carthage</td>\n",
       "      <td>314921</td>\n",
       "      <td>10.318000</td>\n",
       "      <td>36.850032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Cisalpine Gaul</td>\n",
       "      <td>Is</td>\n",
       "      <td>894008</td>\n",
       "      <td>42.821500</td>\n",
       "      <td>33.643600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Comum</td>\n",
       "      <td>Comum</td>\n",
       "      <td>383627</td>\n",
       "      <td>9.085784</td>\n",
       "      <td>45.811920</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Corduba</td>\n",
       "      <td>Corduba</td>\n",
       "      <td>256128</td>\n",
       "      <td>-4.779171</td>\n",
       "      <td>37.884683</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Ebro Valley, Spain</td>\n",
       "      <td>Valley</td>\n",
       "      <td>187606</td>\n",
       "      <td>11.779180</td>\n",
       "      <td>47.892828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Gadiz</td>\n",
       "      <td>I</td>\n",
       "      <td>410517750</td>\n",
       "      <td>14.492318</td>\n",
       "      <td>40.749678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Hippo Regius</td>\n",
       "      <td>Hippo Regius</td>\n",
       "      <td>305090</td>\n",
       "      <td>7.751272</td>\n",
       "      <td>36.882478</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Iliberri</td>\n",
       "      <td>Iliberri</td>\n",
       "      <td>265921</td>\n",
       "      <td>-3.597934</td>\n",
       "      <td>37.176484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Milan</td>\n",
       "      <td>Milan</td>\n",
       "      <td>383706</td>\n",
       "      <td>9.188060</td>\n",
       "      <td>45.463746</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Neapolis</td>\n",
       "      <td>Neapolis</td>\n",
       "      <td>894133</td>\n",
       "      <td>40.564031</td>\n",
       "      <td>35.379381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>North Africa</td>\n",
       "      <td>Afri</td>\n",
       "      <td>314855</td>\n",
       "      <td>9.750000</td>\n",
       "      <td>36.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Parthenope/Neapolis</td>\n",
       "      <td>Parthenope/Neapolis</td>\n",
       "      <td>433014</td>\n",
       "      <td>14.252870</td>\n",
       "      <td>40.839995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Patavium</td>\n",
       "      <td>Patavium</td>\n",
       "      <td>393473</td>\n",
       "      <td>11.876975</td>\n",
       "      <td>45.409561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Pydna</td>\n",
       "      <td>Pydna</td>\n",
       "      <td>491704</td>\n",
       "      <td>22.750000</td>\n",
       "      <td>40.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Roma</td>\n",
       "      <td>Roma</td>\n",
       "      <td>423025</td>\n",
       "      <td>12.486137</td>\n",
       "      <td>41.891775</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Rudiae</td>\n",
       "      <td>Rudiae</td>\n",
       "      <td>442753</td>\n",
       "      <td>18.152294</td>\n",
       "      <td>40.325182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>Sarsina</td>\n",
       "      <td>Sarsina</td>\n",
       "      <td>413306</td>\n",
       "      <td>12.142155</td>\n",
       "      <td>43.919400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>Stridon</td>\n",
       "      <td>Stridon</td>\n",
       "      <td>203160</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>Sulmo</td>\n",
       "      <td>Sulmo</td>\n",
       "      <td>438842</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>Thagaste</td>\n",
       "      <td>Thagaste</td>\n",
       "      <td>315198</td>\n",
       "      <td>8.004957</td>\n",
       "      <td>36.238412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>Ticinum</td>\n",
       "      <td>Ticinum</td>\n",
       "      <td>383798</td>\n",
       "      <td>9.156562</td>\n",
       "      <td>45.185899</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>Tomis</td>\n",
       "      <td>Tomis</td>\n",
       "      <td>756668</td>\n",
       "      <td>30.750000</td>\n",
       "      <td>27.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>Velitrae</td>\n",
       "      <td>Velitrae</td>\n",
       "      <td>423117</td>\n",
       "      <td>12.777210</td>\n",
       "      <td>41.687426</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>Venusia</td>\n",
       "      <td>Venusia</td>\n",
       "      <td>442847</td>\n",
       "      <td>15.827458</td>\n",
       "      <td>40.968880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>Verona</td>\n",
       "      <td>Verona</td>\n",
       "      <td>383816</td>\n",
       "      <td>10.994371</td>\n",
       "      <td>45.438954</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  place                match pleiades_id        lat       long\n",
       "0            Alexandria           Alexandria      727070  29.909773  31.201435\n",
       "1             Amiternum            Amiternum      413013  13.306221  42.400467\n",
       "2                 Andes                Andes      383570  10.824999  45.125167\n",
       "3             Antiochia            Antiochia      897709        NaN        NaN\n",
       "4                Aponus               Aponus      393376  11.783797  45.331858\n",
       "5               Aquinum              Aquinum      432695  13.705626  41.492208\n",
       "6                 Arles                Arles      148217   4.630799  43.677616\n",
       "7               Arpinum              Arpinum      432700  13.613727  41.648358\n",
       "8              Asisium               Asisium      413037  12.618957  43.070619\n",
       "9   Augusta Taurinorum    Augusta Taurinorum      383580   7.685998  45.071779\n",
       "10                Autun                Autun      177460   4.299177  46.947240\n",
       "11             Bilbilis             Bilbilis      246247  -1.603219  41.382010\n",
       "12            Bithynia              Bithynia      991389  32.500000  42.500000\n",
       "13          Brigantium            Brigantium      236465  -8.395968  43.371485\n",
       "14           Calagurris              La(...)      711243        NaN        NaN\n",
       "15             Carthage             Carthage      314921  10.318000  36.850032\n",
       "16       Cisalpine Gaul                   Is      894008  42.821500  33.643600\n",
       "17                Comum                Comum      383627   9.085784  45.811920\n",
       "18              Corduba              Corduba      256128  -4.779171  37.884683\n",
       "19   Ebro Valley, Spain               Valley      187606  11.779180  47.892828\n",
       "20               Gadiz                     I   410517750  14.492318  40.749678\n",
       "21        Hippo Regius          Hippo Regius      305090   7.751272  36.882478\n",
       "22             Iliberri             Iliberri      265921  -3.597934  37.176484\n",
       "23                Milan                Milan      383706   9.188060  45.463746\n",
       "24             Neapolis             Neapolis      894133  40.564031  35.379381\n",
       "25         North Africa                 Afri      314855   9.750000  36.750000\n",
       "26  Parthenope/Neapolis  Parthenope/Neapolis      433014  14.252870  40.839995\n",
       "27             Patavium             Patavium      393473  11.876975  45.409561\n",
       "28                Pydna                Pydna      491704  22.750000  40.250000\n",
       "29                 Roma                 Roma      423025  12.486137  41.891775\n",
       "30               Rudiae               Rudiae      442753  18.152294  40.325182\n",
       "31              Sarsina              Sarsina      413306  12.142155  43.919400\n",
       "32              Stridon              Stridon      203160        NaN        NaN\n",
       "33                Sulmo                Sulmo      438842        NaN        NaN\n",
       "34            Thagaste              Thagaste      315198   8.004957  36.238412\n",
       "35              Ticinum              Ticinum      383798   9.156562  45.185899\n",
       "36               Tomis                 Tomis      756668  30.750000  27.750000\n",
       "37            Velitrae              Velitrae      423117  12.777210  41.687426\n",
       "38              Venusia              Venusia      442847  15.827458  40.968880\n",
       "39               Verona               Verona      383816  10.994371  45.438954"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Put into dataframe\n",
    "\n",
    "headers = ['place', 'match', 'pleiades_id', 'lat', 'long']\n",
    "df = pd.DataFrame(rows, columns=headers)\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export as csv\n",
    "\n",
    "df.to_csv('places.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Development notes:\n",
    "- ~~Plain listing of 'Alexandria' is tricky; 12 different Alexandria in the Pleiades geojson data. An argument for using URIs. For this, could override choices like Alexandria, Antiocha, etc.~~\n",
    "- Look at Sulmo; two 100 matches, but only want to keep the one with coordinate data; also Stridon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'Sulmo' in choices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Sulmo', 100),\n",
       " ('Sulmo', 100),\n",
       " ('M(...)', 90),\n",
       " ('[ - ? - ]o', 90),\n",
       " ('S(...)', 90)]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "process.extract('Sulmo', choices)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
